<?php
#################################
# Metadata Enhancement for Repositories (MER) 
# This software is an overlay repository search interface that uses a 
# text mined index of terms.
# Copyright (C) 2011  Rory McNicholl (University of London Computer Centre), Robert Drinkall (University College London)

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Contact repositories@ulcc.ac.uk
# http://www.ulcc.ac.uk

##################################

class config{
	//scripts
	public $file = "/usr/bin/file";
	public $perl = "/usr/bin/perl";
	public $php = "/usr/bin/php";
	public $antiword = "/usr/local/bin/antiword";
	public $pdftotext = "/usr/bin/pdftotext";

	//db auth
	public $host = "localhost";
	public $username = "mer";
	public $password =  "";
	public $dbname = "mer";
	public $table_prefix = "mer";
	//locations
	public $http_root = "/merlin";
	//repo details (needs some automation!)
	public $repo_sw = "eprints";
	public $repo_id = "myrepo";
	public $repo_id = "My Repository";
	public $repo_url = "http://myrepo.ac.uk";
	public $repo_path = "/opt/eprints3";
	public $repo_version = 3.2;
	public $repo_oai_id = "oai:generic.eprints.org:"; #item id will be appended to this string
	public $ep_full_text_set_id = ""; #EPrints only (afaik) The OAI set id for full_text == public, you may need to set this set up in oai.pl 

	//termine service (you will need a key from http://www.nactem.ac.uk, free for ac domains)
	public $termine = "http://www.nactem.ac.uk/software/termine/webservice/termine.wsdl";

	//settings and other config
	public $threshold = 1; //defaults to 1

}


######### EDIT BELOW THIS LINE AT YOUR OWN RISK #########

class MER
{
	// property declaration
	public $config;
	public $db;
	public $error = array();
	public $sf = array();
	public $reqs = array("curl", "libxml", "soap");
	public $curr_id;
	public $record_count;

	function __construct() {
		$this->config = new config();

       		if(!$this->dbconnect()){
			error_log(array_pop($this->error));
		}
   	}
	public function dbconnect(){

		if(!$this->config->host || !$this->config->username || !$this->config->password){
			array_push($this->error,"Missing database parameters please update config class");
			return false;
		}
		$this->db = @mysql_connect($this->config->host,$this->config->username,$this->config->password);
		if(!$this->db){
			array_push($this->error,"Could not connect to server. Have you created the user '{$this->config->username}'?");
			return false;
		}
		$db_select = @mysql_select_db($this->config->dbname);
		if(!$db_select){
			array_push($this->error,"could not select the database '{$this->config->dbname}'");
			return false;
		}
		return true;
	}
	public function get_search_fields(){

		$mer_get_sf = sprintf("select search_fields from {$this->config->table_prefix}_meta where repository = '%s'", $this->mres($this->config->repo_id));
		$result = mysql_query($mer_get_sf,$this->db);
		$row = mysql_fetch_row($result);
		$sf = split(",", $row[0]);
		foreach($sf as $v){
			$this->sf[$v]=1;
		}
	}
	public function get_threshold(){

		$mer_get_th = sprintf("select threshold from {$this->config->table_prefix}_meta where repository = '%s'", $this->mres($this->config->repo_id));
		$result = mysql_query($mer_get_th,$this->db);
		$row = mysql_fetch_row($result);
		if($row){
			$this->config->threshold = $row[0];
			error_log($this->config->threshold);
		}
		return $this->config->threshold;
	}
	public function get_PID(){

		$mer_get_PID = sprintf("select PID, harvest_log from {$this->config->table_prefix}_meta where repository = '%s'", $this->mres($this->config->repo_id));
		$result = mysql_query($mer_get_PID,$this->db);
		$row = mysql_fetch_assoc($result);
		if($row){
			return array($row["PID"], $row["harvest_log"]);
		}else{
			return false;
		}
	}
	public function remove_PID($mypid){

		$mer_remove_PID = sprintf("update {$this->config->table_prefix}_meta set PID = NULL where repository = '%s' and PID = $mypid", $this->mres($this->config->repo_id));
		$result = mysql_query($mer_remove_PID,$this->db);
		#TODO error catching
		return true;
	}


	public function mres($str){
		return mysql_real_escape_string($str);
	}

	public function message($str){
		error_log("{$this->record_count} [{$this->curr_id}] $str");
	}

	public function do_cURL($url, $method="GET", $data=false){
		// Invoke the Web service via HTTP POST
		$ch = curl_init();

		curl_setopt($ch, CURLOPT_URL, $url);
		curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
		curl_setopt($ch, CURLOPT_HEADER, 0);
		if($method =="POST" && $data){
			curl_setopt($ch, CURLOPT_POSTFIELDS, $data);
			curl_setopt($ch, CURLOPT_POST, 1);
		}else{
			curl_setopt($ch, CURLOPT_HTTPGET, 1);
		}
		curl_setopt($ch, CURLOPT_TIMEOUT, 60);
		$response = curl_exec($ch); 

		if (curl_errno($ch) != 0){
			message(curl_error($ch));
		}
		curl_close($ch);

		return $response;

	}

	public function get_EP_field_menu($field=false){
		$field_script = $this->config->repo_url."/cgi/finput";
		if($field) $field_script .="/".$field;
		$divs = file( $field_script );
		array_shift($divs);
		return join(" ",$divs);
	}
	
	public function check_requirements(){

		$missing = false;
		$loaded = array();
		foreach($this->reqs as $req){
			if(!extension_loaded($req)){
				$missing = true;
				$loaded[$req]=false;
			}else{
				$loaded[$req]=true;
			}
		}
		if($missing){
			print "<p>The following required PHP extensions are not loaded:</p><ul>";
			foreach($reqs as $req){
				print "<li>$req "; 
				$loaded[$req] ? print "[<span style='color:#0F0;'>OK</span>]" : print "[<span style='color:#f00;'>FAIL</span>]";
				print "</li>";
			}
			print "</ul>";
			print "<p>Please install the required extensions and try again.</p>";

			exit;
		}
#		$db = dbconnect();
		if(!$this->db){
			if(isset($_REQUEST["mr"])){
				$this->db = $this->db_create();
				if(!$this->db){
					print "<p>Database related problem: <span style='color:#f00;'>".array_pop($this->error)."</span></p>";
					exit;
				}
			}else{
				print "<p>If you can log on as Mysql root and create the database: </p>";
				?>
					<form action="" method="post">
					<label for="mr">MySQL admin username:</label>
					<input id="mr" type="text" name="mr"/>
					<label for="rp">MySQL admin password:</label>
					<input id="rp" type="password" name="rp"/>
					<input type="submit" value="Create database and DB user and tables"/>
					</form>
				<?php
				exit;
			}
		}
		if(!$this->check_tables()){
			if(!$this->create_tables()) exit;
		}
		if(!$this->check_harvester()){
			print "<p>Harvester related problem: <span style='color:#f00;'>".array_pop($this->error)."</span></p>";
			exit;
		}
	}
	public function check_harvester(){

		if(!file_exists($this->config->file))
			array_push($mer_error,"file not in {$this->config->file}");


		try {
			$T_client = new SoapClient($this->config->termine);
			$termine_output = $T_client->analyze("The quick brown fox jumped over the lazy imagination.");
		} catch (SoapFault $fault) {
			array_push($this->error,$fault);
			return false;
		}
		return true;
	}
	public function check_tables(){

		$result = mysql_query("show tables",$this->db);
		if(!$result){
			array_push($this->error,"Couldn't show tables");
			return false;

		}
		if(!mysql_num_rows($result)){
			return false;
		}
		return true;

	}
	public function create_tables(){

		$mer_assoc_create = "CREATE TABLE `{$this->config->table_prefix}_associations` (
					`assoc_id` int(11) NOT NULL auto_increment,
					`record_id` varchar(255) default NULL,
					`termine_score` float default NULL,
					`in_metadata` tinyint(4) default NULL,
					`term_id` int(11) default NULL,
					`user_score` int(11) default NULL,
					PRIMARY KEY  (`assoc_id`),
					KEY `record` (`record_id`),
					KEY `term_id` (`term_id`),
					KEY `termine_score` (`termine_score`)
					) DEFAULT CHARSET=utf8 ";
		$mer_terms_create = "CREATE TABLE `{$this->config->table_prefix}_terms` (
					`term_id` int(11) NOT NULL auto_increment,
					`term` text,
					`num_docs` int(11) default NULL,
					`lasso_average` float default NULL,
					PRIMARY KEY  (`term_id`),
					KEY `term` (`term`(32))
					) DEFAULT CHARSET=utf8";

		$mer_records_create = "CREATE TABLE `{$this->config->table_prefix}_records` (
					`record_id` varchar(255) NOT NULL,
					`timestamp` int(11) default NULL,
					`num_assoc` int(11) default NULL,
					`update` tinyint NULL,
					PRIMARY KEY  (`record_id`)
					) DEFAULT CHARSET=utf8";

		$mer_metadata_create = "CREATE TABLE `{$this->config->table_prefix}_meta` (
					`meta_id` int(11) NOT NULL auto_increment,
					`repository` text NULL,
					`search_fields` text NULL,
					`threshold` int(11) NULL,
					`PID` int(11) NULL,
					`harvest_log` text NULL,
					PRIMARY KEY  (`meta_id`)
					) DEFAULT CHARSET=utf8";
		//init the mer_meta table with repo id and termine threshold
		$mer_metadata_init = "INSERT INTO `{$this->config->table_prefix}_meta` (`repository`, `threshold`) values ('{$this->config->repo_id}', 1)";

		$result1 = mysql_query($mer_assoc_create,$this->db);
		$result2 = mysql_query($mer_terms_create, $this->db);
		$result3 = mysql_query($mer_records_create, $this->db);
		$result4 = mysql_query($mer_metadata_create, $this->db);
		$result5 = mysql_query($mer_metadata_init, $this->db);

		if(!$result1 || !$result2 || !$result3 || !$result4 || !$result5){
			print "<p><span style='color:#f00;'>Some database tables not present and could not create them... you have a go:</span></p>";
			if(!$result1)
				print "<pre>$mer_assoc_create</pre>";
			if(!$result2)
				print "<pre>$mer_terms_create</pre>";
			if(!$result3)
				print "<pre>$mer_records_create</pre>";
			if(!$result4)
				print "<pre>$mer_metadata_create</pre>";
			if(!$result5)
				print "<pre>$mer_metadata_init</pre>";

			return false;
		}
		print "<p><span style='color:#0f0;'>Database tables have been created</span></p>";
		return true;
	}
	public function mer_stats(){

		$result = mysql_query("select count(distinct record_id) as records from {$this->config->table_prefix}_associations");
		list($records_processed) = mysql_fetch_row($result);

		$result = mysql_query("select count(*) from {$this->config->table_prefix}_associations");
		list($assocs) = mysql_fetch_row($result);

		$result = mysql_query("select count(*) from {$this->config->table_prefix}_terms");
		list($terms) = mysql_fetch_row($result);

		return array($terms, $assocs, $records_processed);
	}
	public function db_create(){
		global $_REQUEST;

		if(preg_match('/[;,\'"]/',$_REQUEST['mr'])){
			array_push($this->error,"Illegal characters in DB admin username.");
			return false;
		}
		if(preg_match('/[;,\'"]/',$_REQUEST['rp'])){
			array_push($this->error,"Illegal characters in DB admin password.");
			return false;
		}

		$rdb = @mysql_connect($this->config->host,$_REQUEST["mr"],$_REQUEST["rp"]);
		if(!$rdb){
			array_push($this->error,"Could not connect to server.");
			return false;
		}
		$mer_db_create = sprintf("CREATE DATABASE IF NOT EXISTS `%s` DEFAULT CHARSET=utf8", $this->mres($this->config->dbname));
		$result1 = mysql_query($mer_db_create,$rdb);

		$mer_user_create = sprintf("CREATE USER '%s'@'%s' IDENTIFIED BY '%s'", 
			$this->mres($this->config->username),
			$this->mres($this->config->host),
			$this->mres($this->config->password));
		$result2 = mysql_query($mer_user_create,$rdb);
			
		$mer_user_grant = sprintf("GRANT ALL PRIVILEGES ON %s.* TO '%s'@'%s'",
			$this->mres($this->config->dbname),
			$this->mres($this->config->username),
			$this->mres($this->config->host));

		$result3 = mysql_query($mer_user_grant,$rdb);

		$mer_user_grant = sprintf("GRANT SELECT ON %s.* TO '%s'@'%s'",
			$this->mres($this->config->repo_id),
			$this->mres($this->config->username),
			$this->mres($this->config->host));

		$result3 = mysql_query($mer_user_grant,$rdb);

		if(!$result1 || !$result2 || !$result3){
			print "<p><span style='color:#f00;'>Problem creating database and user... you have a go:</span></p>";
			if(!$result1)
				print "<pre>$mer_db_create</pre>";
			if(!$result2)
				print "<pre>$mer_user_create</pre>";
			if(!$result3)
				print "<pre>$mer_user_grant</pre>";

			return false;
		}

		print "<p><span style='color:#0f0;'>Database and user have been created</span></p>";
		mysql_close ( $rdb );
		
		return $this->dbconnect();
	}

	public function save_fields(){
		global $_POST;

		$sf_str = "";
		foreach($_POST as $k=>$v){
			if($k == "save_fields"){
				continue;
			}
			$sf_str .="$k,";
		}
		$sf_str = substr($sf_str,0,-1);
		$mer_save_sf = sprintf("update {$this->config->table_prefix}_meta set search_fields = '%s' where repository = '%s'", $this->mres($sf_str), $this->mres($this->config->repo_id));
		$result = mysql_query($mer_save_sf,$this->db);

		return true;
	}
	public function save_threshold(){
		global $_POST;

		$mer_save_th = sprintf("update {$this->config->table_prefix}_meta set threshold = '%s' where repository = '%s'", $this->mres($_POST["threshold"]), $this->mres($this->config->repo_id));
		
		$result = mysql_query($mer_save_th,$this->db);
		if(!$result) return false;

		print "<p><span style='color:#0f0;'>TerMine threshold has been updated.</span></p>";
		return true;

	}
	public function save_PID($PID, $logfile){

		$mer_save_PID = sprintf("update {$this->config->table_prefix}_meta set PID = %d, harvest_log = '%s' where repository = '%s'", $PID, $this->mres($logfile), $this->mres($this->config->repo_id));
		
		$result = mysql_query($mer_save_PID,$this->db);
		if(!$result) return false;

		return true;

	}

	public function harvest(){
		global $_SERVER;
		
		//run it in the background... need way to check...
		$cmd = "{$this->config->php} {$_SERVER["DOCUMENT_ROOT"]}/{$this->config->http_root}/harvester/harvester.php {$this->config->threshold}";
		$tmpfname = tempnam("/tmp", "MER_harvest_log_");
		exec("$cmd 2>$tmpfname >&- /dev/null & echo $!", $out, $ret);
		$PID = $out[0];
		$this->save_PID($PID, $tmpfname);
		print "<p><span style='color:#0f0;'>The term harvester has been started (PID: $PID). Ingest is being logged in $tmpfname.</span></p>";

		return true;
	}

	public function clear_terms(){

		$mer_del_terms = sprintf("delete from {$this->config->table_prefix}_terms");
		$mer_del_records = sprintf("delete from {$this->config->table_prefix}_records");
		$mer_del_assoc = sprintf("delete from {$this->config->table_prefix}_associations");
		
		$result1 = mysql_query($mer_del_terms,$this->db);
		$result2 = mysql_query($mer_del_records,$this->db);
		$result3 = mysql_query($mer_del_assoc,$this->db);

		if(!$result1 || !$result2 || !$result3){
			print "<p><span style='color:#f00;'>Could not clear terms.</span></p>";
			return false;
		}
		print "<p><span style='color:#0f0;'>All terms and associations have been removed.</span></p>";

		return true;
	}

	######## Harvester #####
	public function record_already(){

		$SQL  = sprintf("select * from {$this->config->table_prefix}_records where record_id = '%s'", $this->mres($this->curr_id));
		$result = @mysql_query($SQL, $this->db) or die("could not complete your query: $SQL\n");
		$row = mysql_fetch_assoc($result);
		if(!$row){
			#no result, no mention of record, needs harvesting
			$this->message("no result, no mention of record, needs harvesting");
			return false;
		}else{
			if($row["update"]){
				#have record, but set to update, needs harvesting... remove existing assocs?
				$this->message("have record, but set to update, needs harvesting... remove existing assocs?");
				return false;
			}else{
				#have record, no update flag, no need for harvesting
				$this->message("have record, no update flag, no need for harvesting");
				return true;
			}
		}
	}
	public function get_digi_objs($dc){

		$do_data = Array();
		$dc_identifiers = $dc->getElementsByTagName("identifier"); #This is where EPrints puts its filenames...
		foreach($dc_identifiers as $dc_id){
			$candi_uri = $dc_id->nodeValue;
			#two tests to identify whether the dc:identifier is indeed the address of the full-text
			if(preg_match('#^http://#', $candi_uri) && preg_match('#'.quotemeta($this->config->repo_url).'#', $candi_uri)){
				$tmpfname = tempnam("/tmp", "MER_");
				$src = fopen($candi_uri, 'r');
				$dest = fopen($tmpfname, 'w+');
				chmod($tmpfname,0664);
				stream_copy_to_stream($src, $dest);
				$out = array();
				exec("/usr/bin/file -bi $tmpfname",$out,$ret);
				$size = filesize($tmpfname);
				if($ret==0){
					 $mimetype = $out[0];
				}
				$do_data[$this->curr_id]  = array("uri" => $candi_uri,
					 "mimetype" => $mimetype,
					 "size" => $size,
					 "item_id" => $this->curr_id,
					 "bitstream" => $tmpfname);
			}
		}
		return $do_data;
	}
	public function insert_terms($fd){

		$uri = $fd["uri"];

		$error = $this->textify($fd["bitstream"],$fd["mimetype"],$text_file);
		if($error){
			if(strstr($fd["mimetype"],'html') && !strstr($fd["uri"],'htm'))
				$error = "Possible bitstream-uri mismatch (uri: ".$fd['mimetype']."). This in turn may be due to uri not being accessable";
			return array(-1,$error);
		}
		
		$termine_url = $this->config->repo_url."/merlin/harvester/termine_proxy.php?file=$text_file&r_id={$this->curr_id}";
		$termine_xml = $this->do_cURL($termine_url);
		
		$tdoc = new DOMDocument();
		try {
			$tdoc->loadXML( $termine_xml, LIBXML_NOERROR );
		} catch(Exception $e) {
			return array(-3,$e->getMessage());
		}
		$tcs = $tdoc->getElementsByTagName('termCandidate');
		if(!$tcs->length){
			$error = $tdoc->getElementsByTagName('error');
			if(!$error->length)
				return array(-4,"No termCandidates found");
			else
				return array(-5,$error->item(0)->nodeValue);
		}

		$terms_found  = $tcs->length;
		$terms=array();
		for($j=0; $j<$tcs->length; $j++){
			$cvalue = $tcs->item($j)->attributes->getNamedItem("cValueScore")->nodeValue;
			$term = $tcs->item($j)->nodeValue;
			if($cvalue>=$this->config->threshold){
				$terms[$term] = $cvalue;
			}
		}
		asort($terms);
		$terms = array_reverse($terms);
		$num_terms=0;
		$last = 0;
		foreach($terms as $term => $score){
			$term = $this->mres($term);

			if(preg_match('/\b(\w+)\s\1\b/',$term)){
	#			error_log("term ($term) contains REPEATS"); 
				continue;
			}
			$term_id = $this->term_exists($term);
			$in_meta = $this->is_in_meta($term);
			$assoc_id = $this->assoc_exists($term_id);
			if(!$assoc_id){
				$SQL = sprintf("insert into {$this->config->table_prefix}_associations (term_id, record_id, termine_score, in_metadata) values (%d, '%s', %d, %d)",
						$term_id, 
						$this->mres($this->curr_id),
						$score, 
						$in_meta);
				$result = @mysql_query($SQL, $this->db) or die("could not complete your query: $SQL\n");
				$num_terms++;
			}
			$last = $term;
		}
		return array($terms_found,$num_terms);
	}

	//returns text version of file
	public function textify($file,$mimetype,&$text_file){

		$out = array();
		$text_file = $file.".txt";
		#message("printing ascii version of $file to $text_file");
		$ret = -1;
		#message("MIMETYPE: $mimetype");
		$loc = $_SERVER['SCRIPT_FILENAME'];
		$loc = preg_replace('/harvester\.php$/', "", $loc);
		switch($mimetype){
			case "application/msword" : { $cmd = "{$this->config->antiword} ".$file." > ".$text_file; break;}
	#		case "application/pdf" : { $cmd = "/usr/bin/pdftotext ".$file." 2>/dev/null"; break;}
			case "application/pdf" : { $cmd = "{$this->config->perl} $loc/strip_pdf ".$file." 2> /dev/null > ".$text_file; break; }
#			case "application/pdf" : { $cmd = "{$this->config->perl} ./strip_pdf ".$file." 2> /dev/null > ".$text_file; break; }

			default : { $cmd = false; break;}
		}
		$this->message($cmd);
		if($cmd)
			exec($cmd,$out,$ret);
		else
			return $mimetype;
		
		//test resultant text file...
		$t_cmd = "{$this->config->file} \"$text_file\"";
		exec($t_cmd,$t_out,$t_ret);
		if(preg_match("/ASCII text, with no line terminators/",join(" ",$t_out)))
			return "The resultant text file ($text_file) doesn't look like much cop";

		return $ret;
	}
	
	public function insert_record($num_assoc){
		
		$SQL  = sprintf("insert into {$this->config->table_prefix}_records (record_id, timestamp, num_assoc) values ('%s', %s, %d)", $this->mres($this->curr_id), date('YmdHis'), $num_assoc);
		$result = @mysql_query($SQL, $this->db) or die("could not complete your query: $SQL\n");
	}


	public function term_exists($term){

		$SQL  = "select term_id from {$this->config->table_prefix}_terms where term = '".$this->mres($term)."'";
		$result = @mysql_query($SQL, $this->db) or die("could not complete your query: $SQL\n");
		if($row = mysql_fetch_assoc($result)){
			return $row["term_id"];
		}else{
			$SQL  = "insert into {$this->config->table_prefix}_terms (term) values ('".$this->mres($term)."')";
			$result = @mysql_query($SQL, $this->db) or die("could not complete your query: $SQL\n");
			return mysql_insert_id();
		}
	}

	public function is_in_meta($term){
		$bits  = split(":",$this->curr_id);
		$e_id = array_pop($bits);
		$SQL  = sprintf("select count(*) as meta_hits from %s.eprint where eprintid = %d and (abstract like '%%%s%%' or title like '%%%s%%')", $this->mres($this->config->repo_id), $e_id, $this->mres($term), $this->mres($term));
		
		$result = @mysql_query($SQL, $this->db) or die("could not complete your query: $SQL\n");
		$in_meta=0;
		while($row = mysql_fetch_assoc($result)){
			$in_meta = $row["meta_hits"];
		}
		return $in_meta;
	}
	public function assoc_exists($term_id){

		$SQL  = sprintf("select assoc_id from {$this->config->table_prefix}_associations where term_id = %d and record_id = '%s'", $term_id, $this->mres($this->curr_id));

		$result = @mysql_query($SQL, $this->db) or die("could not complete your query: $SQL\n");
		$assoc_id=false;
		while($row = mysql_fetch_assoc($result)){
			$assoc_id = $row["assoc_id"];
		}
		return $assoc_id;
	}

}
?>
